"""Configuration Template Management System.

Provides comprehensive template management functionality including template
creation, validation, version control, import/export, and application to devices.
"""

import asyncio
import gzip
import hashlib
import json
import logging
import os
import shutil
import time
import uuid
import yaml
from dataclasses import dataclass, field, asdict
from datetime import datetime, timedelta
from enum import Enum
from pathlib import Path
from typing import Dict, List, Optional, Any, Tuple, Set
from concurrent.futures import ThreadPoolExecutor

logger = logging.getLogger(__name__)


# Configuration Constants
TEMPLATE_CONFIG = {
    "max_templates": 100,
    "max_template_size": 10240,
    "version_history_limit": 10,
    "auto_validation": True,
    "backup_enabled": True,
    "compression": True,
    "encryption": False,
    "sync_interval": 300,
    "cache_size": 50,
    "cache_ttl": 3600
}


class TemplateCategory(Enum):
    """Template categories."""
    NETWORK = "network"
    SYSTEM = "system" 
    APPLICATION = "application"
    SECURITY = "security"
    COMPOSITE = "composite"
    CUSTOM = "custom"


class TemplateStatus(Enum):
    """Template status."""
    DRAFT = "draft"
    ACTIVE = "active"
    DEPRECATED = "deprecated"
    ARCHIVED = "archived"
    ERROR = "error"


class TemplateOperation(Enum):
    """Template operations."""
    CREATE = "create"
    UPDATE = "update"
    DELETE = "delete"
    APPLY = "apply"
    EXPORT = "export"
    IMPORT = "import"
    VALIDATE = "validate"


@dataclass
class ValidationRule:
    """Validation rule definition."""
    field_path: str
    rule_type: str
    rule_params: Dict[str, Any]
    error_message: str
    severity: str = "error"


@dataclass
class TemplateVersion:
    """Template version information."""
    version: str
    timestamp: datetime
    author: str
    changes: List[str] = field(default_factory=list)
    config_diff: Dict[str, Any] = field(default_factory=dict)
    is_major: bool = False


@dataclass
class ConfigTemplate:
    """Configuration template data model."""
    template_id: str
    name: str
    description: str = ""
    category: TemplateCategory = TemplateCategory.CUSTOM
    tags: List[str] = field(default_factory=list)
    version: str = "1.0.0"
    author: str = "system"
    created_at: datetime = field(default_factory=datetime.now)
    updated_at: datetime = field(default_factory=datetime.now)
    status: TemplateStatus = TemplateStatus.DRAFT
    
    # Configuration content
    config_schema: Dict[str, Any] = field(default_factory=dict)
    default_values: Dict[str, Any] = field(default_factory=dict)
    validation_rules: List[ValidationRule] = field(default_factory=list)
    
    # Metadata
    device_compatibility: List[str] = field(default_factory=list)
    firmware_requirements: Dict[str, str] = field(default_factory=dict)
    dependencies: List[str] = field(default_factory=list)
    usage_count: int = 0
    success_rate: float = 0.0
    
    # Version control
    parent_template_id: Optional[str] = None
    version_history: List[TemplateVersion] = field(default_factory=list)
    is_locked: bool = False


@dataclass
class ValidationResult:
    """Validation result."""
    is_valid: bool
    errors: List[str] = field(default_factory=list)
    warnings: List[str] = field(default_factory=list)
    suggestions: List[str] = field(default_factory=list)


@dataclass
class CompatibilityResult:
    """Compatibility check result."""
    is_compatible: bool
    missing_features: List[str] = field(default_factory=list)
    version_conflicts: List[str] = field(default_factory=list)
    recommendations: List[str] = field(default_factory=list)


@dataclass
class TemplateApplicationContext:
    """Template application context."""
    template: ConfigTemplate
    target_devices: List[str]
    override_values: Dict[str, Any] = field(default_factory=dict)
    application_options: Dict[str, Any] = field(default_factory=dict)
    rollback_data: Dict[str, Any] = field(default_factory=dict)
    
    # Application state
    status: str = "pending"
    progress: float = 0.0
    errors: List[str] = field(default_factory=list)
    warnings: List[str] = field(default_factory=list)


@dataclass
class TemplateApplicationResult:
    """Template application result."""
    context: TemplateApplicationContext
    success: bool
    applied_devices: List[str] = field(default_factory=list)
    failed_devices: List[str] = field(default_factory=list)
    rollback_required: bool = False


class TemplateCache:
    """Template caching system."""
    
    def __init__(self, max_size: int = 50, ttl: int = 3600):
        """Initialize template cache.
        
        Args:
            max_size: Maximum cache size
            ttl: Time to live in seconds
        """
        self.max_size = max_size
        self.ttl = ttl
        self.cache: Dict[str, Tuple[ConfigTemplate, float]] = {}
        self.access_times: Dict[str, float] = {}
        
    def get(self, template_id: str) -> Optional[ConfigTemplate]:
        """Get template from cache.
        
        Args:
            template_id: Template identifier
            
        Returns:
            Cached template or None
        """
        if template_id not in self.cache:
            return None
        
        template, cached_time = self.cache[template_id]
        
        # Check TTL
        if time.time() - cached_time > self.ttl:
            self.invalidate(template_id)
            return None
        
        # Update access time
        self.access_times[template_id] = time.time()
        return template
    
    def put(self, template: ConfigTemplate):
        """Put template into cache.
        
        Args:
            template: Template to cache
        """
        # Evict if cache is full
        if len(self.cache) >= self.max_size:
            self._evict_lru()
        
        current_time = time.time()
        self.cache[template.template_id] = (template, current_time)
        self.access_times[template.template_id] = current_time
    
    def invalidate(self, template_id: str):
        """Invalidate cached template.
        
        Args:
            template_id: Template identifier
        """
        self.cache.pop(template_id, None)
        self.access_times.pop(template_id, None)
    
    def clear(self):
        """Clear all cached templates."""
        self.cache.clear()
        self.access_times.clear()
    
    def _evict_lru(self):
        """Evict least recently used template."""
        if not self.access_times:
            return
        
        # Find LRU template
        lru_template_id = min(self.access_times.items(), key=lambda x: x[1])[0]
        self.invalidate(lru_template_id)


class TemplateValidator:
    """Template validation system."""
    
    def __init__(self):
        """Initialize template validator."""
        self.built_in_rules = self._load_built_in_rules()
        
    def _load_built_in_rules(self) -> Dict[str, Any]:
        """Load built-in validation rules."""
        return {
            "required": self._validate_required,
            "type": self._validate_type,
            "length": self._validate_length,
            "range": self._validate_range,
            "pattern": self._validate_pattern,
            "enum": self._validate_enum,
            "format": self._validate_format
        }
    
    def validate_template(self, template: ConfigTemplate) -> ValidationResult:
        """Validate template configuration.
        
        Args:
            template: Template to validate
            
        Returns:
            Validation result
        """
        result = ValidationResult(is_valid=True)
        
        try:
            # Validate template structure
            structure_result = self._validate_template_structure(template)
            result.errors.extend(structure_result.errors)
            result.warnings.extend(structure_result.warnings)
            
            # Validate configuration schema
            schema_result = self._validate_config_schema(template.config_schema)
            result.errors.extend(schema_result.errors)
            result.warnings.extend(schema_result.warnings)
            
            # Validate default values against schema
            values_result = self._validate_values_against_schema(
                template.default_values, template.config_schema
            )
            result.errors.extend(values_result.errors)
            result.warnings.extend(values_result.warnings)
            
            # Validate custom rules
            rules_result = self._validate_custom_rules(
                template.default_values, template.validation_rules
            )
            result.errors.extend(rules_result.errors)
            result.warnings.extend(rules_result.warnings)
            
            result.is_valid = len(result.errors) == 0
            
        except Exception as e:
            result.is_valid = False
            result.errors.append(f"Validation error: {e}")
            logger.error(f"Template validation error: {e}")
        
        return result
    
    def _validate_template_structure(self, template: ConfigTemplate) -> ValidationResult:
        """Validate template structure."""
        result = ValidationResult(is_valid=True)
        
        # Check required fields
        if not template.name.strip():
            result.errors.append("Template name is required")
        
        if not template.template_id:
            result.errors.append("Template ID is required")
        
        # Check template size
        template_size = len(json.dumps(asdict(template)))
        if template_size > TEMPLATE_CONFIG["max_template_size"]:
            result.errors.append(
                f"Template size ({template_size}) exceeds maximum "
                f"({TEMPLATE_CONFIG['max_template_size']})"
            )
        
        # Validate version format
        if not self._is_valid_semver(template.version):
            result.warnings.append(f"Version '{template.version}' is not valid semantic version")
        
        return result
    
    def _validate_config_schema(self, schema: Dict[str, Any]) -> ValidationResult:
        """Validate configuration schema."""
        result = ValidationResult(is_valid=True)
        
        if not schema:
            result.warnings.append("Empty configuration schema")
            return result
        
        # Check schema structure
        if not isinstance(schema, dict):
            result.errors.append("Schema must be an object")
            return result
        
        # Validate schema properties
        for key, value in schema.items():
            if not isinstance(key, str):
                result.errors.append(f"Schema key must be string: {key}")
                continue
            
            if not isinstance(value, dict):
                result.errors.append(f"Schema value must be object: {key}")
                continue
        
        return result
    
    def _validate_values_against_schema(self, values: Dict[str, Any], 
                                      schema: Dict[str, Any]) -> ValidationResult:
        """Validate values against schema."""
        result = ValidationResult(is_valid=True)
        
        if not schema:
            return result
        
        # Check each value against schema
        for key, value in values.items():
            if key not in schema:
                result.warnings.append(f"Value '{key}' not defined in schema")
                continue
            
            schema_def = schema[key]
            if not self._validate_value_type(value, schema_def):
                result.errors.append(f"Value '{key}' type mismatch")
        
        return result
    
    def _validate_custom_rules(self, values: Dict[str, Any],
                             rules: List[ValidationRule]) -> ValidationResult:
        """Validate custom rules."""
        result = ValidationResult(is_valid=True)
        
        for rule in rules:
            try:
                # Get value from path
                value = self._get_value_by_path(values, rule.field_path)
                if value is None:
                    if rule.severity == "error":
                        result.errors.append(f"Required field missing: {rule.field_path}")
                    else:
                        result.warnings.append(f"Optional field missing: {rule.field_path}")
                    continue
                
                # Apply validation rule
                rule_func = self.built_in_rules.get(rule.rule_type)
                if rule_func and not rule_func(value, rule.rule_params):
                    if rule.severity == "error":
                        result.errors.append(rule.error_message)
                    else:
                        result.warnings.append(rule.error_message)
                        
            except Exception as e:
                result.errors.append(f"Rule validation error for {rule.field_path}: {e}")
        
        return result
    
    def _validate_required(self, value: Any, params: Dict[str, Any]) -> bool:
        """Validate required field."""
        return value is not None and value != ""
    
    def _validate_type(self, value: Any, params: Dict[str, Any]) -> bool:
        """Validate value type."""
        expected_type = params.get("type")
        if expected_type == "string":
            return isinstance(value, str)
        elif expected_type == "integer":
            return isinstance(value, int)
        elif expected_type == "number":
            return isinstance(value, (int, float))
        elif expected_type == "boolean":
            return isinstance(value, bool)
        elif expected_type == "array":
            return isinstance(value, list)
        elif expected_type == "object":
            return isinstance(value, dict)
        return True
    
    def _validate_length(self, value: Any, params: Dict[str, Any]) -> bool:
        """Validate length constraint."""
        if not hasattr(value, '__len__'):
            return True
        
        length = len(value)
        min_length = params.get("min", 0)
        max_length = params.get("max", float('inf'))
        
        return min_length <= length <= max_length
    
    def _validate_range(self, value: Any, params: Dict[str, Any]) -> bool:
        """Validate range constraint."""
        if not isinstance(value, (int, float)):
            return True
        
        min_val = params.get("min", float('-inf'))
        max_val = params.get("max", float('inf'))
        
        return min_val <= value <= max_val
    
    def _validate_pattern(self, value: Any, params: Dict[str, Any]) -> bool:
        """Validate pattern constraint."""
        if not isinstance(value, str):
            return True
        
        import re
        pattern = params.get("pattern", ".*")
        return bool(re.match(pattern, value))
    
    def _validate_enum(self, value: Any, params: Dict[str, Any]) -> bool:
        """Validate enum constraint."""
        allowed_values = params.get("values", [])
        return value in allowed_values
    
    def _validate_format(self, value: Any, params: Dict[str, Any]) -> bool:
        """Validate format constraint."""
        if not isinstance(value, str):
            return True
        
        format_type = params.get("format")
        if format_type == "email":
            return "@" in value and "." in value
        elif format_type == "ip":
            parts = value.split(".")
            return (len(parts) == 4 and 
                   all(p.isdigit() and 0 <= int(p) <= 255 for p in parts))
        elif format_type == "mac":
            import re
            return bool(re.match(r'^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$', value))
        return True
    
    def _get_value_by_path(self, data: Dict[str, Any], path: str) -> Any:
        """Get value by dotted path."""
        keys = path.split(".")
        current = data
        
        for key in keys:
            if isinstance(current, dict) and key in current:
                current = current[key]
            else:
                return None
        
        return current
    
    def _validate_value_type(self, value: Any, schema_def: Dict[str, Any]) -> bool:
        """Validate value type against schema definition."""
        expected_type = schema_def.get("type")
        if not expected_type:
            return True
        
        return self._validate_type(value, {"type": expected_type})
    
    def _is_valid_semver(self, version: str) -> bool:
        """Check if version is valid semantic version."""
        import re
        pattern = r'^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?$'
        return bool(re.match(pattern, version))


class TemplateSerializer:
    """Template serialization and deserialization."""
    
    SUPPORTED_FORMATS = ["json", "yaml"]
    
    def export_template(self, template: ConfigTemplate, 
                       format: str = "json", compress: bool = False) -> bytes:
        """Export template to bytes.
        
        Args:
            template: Template to export
            format: Export format (json, yaml)
            compress: Whether to compress the output
            
        Returns:
            Serialized template data
        """
        try:
            # Convert template to dict
            template_dict = asdict(template)
            
            # Convert datetime objects to ISO strings
            template_dict = self._serialize_datetime_fields(template_dict)
            
            # Serialize based on format
            if format.lower() == "json":
                data = json.dumps(template_dict, indent=2, ensure_ascii=False)
            elif format.lower() == "yaml":
                data = yaml.dump(template_dict, default_flow_style=False, allow_unicode=True)
            else:
                raise ValueError(f"Unsupported format: {format}")
            
            # Convert to bytes
            data_bytes = data.encode('utf-8')
            
            # Compress if requested
            if compress:
                data_bytes = gzip.compress(data_bytes)
            
            return data_bytes
            
        except Exception as e:
            logger.error(f"Template export error: {e}")
            raise
    
    def import_template(self, data: bytes, format: str = "json", 
                       compressed: bool = False) -> ConfigTemplate:
        """Import template from bytes.
        
        Args:
            data: Serialized template data
            format: Data format (json, yaml)
            compressed: Whether data is compressed
            
        Returns:
            Imported template
        """
        try:
            # Decompress if needed
            if compressed:
                data = gzip.decompress(data)
            
            # Decode to string
            data_str = data.decode('utf-8')
            
            # Parse based on format
            if format.lower() == "json":
                template_dict = json.loads(data_str)
            elif format.lower() == "yaml":
                template_dict = yaml.safe_load(data_str)
            else:
                raise ValueError(f"Unsupported format: {format}")
            
            # Convert datetime fields back
            template_dict = self._deserialize_datetime_fields(template_dict)
            
            # Convert enum fields
            template_dict = self._deserialize_enum_fields(template_dict)
            
            # Create template object
            return ConfigTemplate(**template_dict)
            
        except Exception as e:
            logger.error(f"Template import error: {e}")
            raise
    
    def export_templates(self, templates: List[ConfigTemplate], 
                        format: str = "json", compress: bool = False) -> bytes:
        """Export multiple templates.
        
        Args:
            templates: Templates to export
            format: Export format
            compress: Whether to compress
            
        Returns:
            Serialized templates data
        """
        templates_data = []
        for template in templates:
            template_dict = asdict(template)
            template_dict = self._serialize_datetime_fields(template_dict)
            templates_data.append(template_dict)
        
        # Serialize collection
        if format.lower() == "json":
            data = json.dumps(templates_data, indent=2, ensure_ascii=False)
        elif format.lower() == "yaml":
            data = yaml.dump(templates_data, default_flow_style=False, allow_unicode=True)
        else:
            raise ValueError(f"Unsupported format: {format}")
        
        data_bytes = data.encode('utf-8')
        
        if compress:
            data_bytes = gzip.compress(data_bytes)
        
        return data_bytes
    
    def import_templates(self, data: bytes, format: str = "json",
                        compressed: bool = False) -> List[ConfigTemplate]:
        """Import multiple templates.
        
        Args:
            data: Serialized templates data
            format: Data format
            compressed: Whether data is compressed
            
        Returns:
            List of imported templates
        """
        if compressed:
            data = gzip.decompress(data)
        
        data_str = data.decode('utf-8')
        
        if format.lower() == "json":
            templates_data = json.loads(data_str)
        elif format.lower() == "yaml":
            templates_data = yaml.safe_load(data_str)
        else:
            raise ValueError(f"Unsupported format: {format}")
        
        templates = []
        for template_dict in templates_data:
            template_dict = self._deserialize_datetime_fields(template_dict)
            template_dict = self._deserialize_enum_fields(template_dict)
            templates.append(ConfigTemplate(**template_dict))
        
        return templates
    
    def _serialize_datetime_fields(self, data: Dict[str, Any]) -> Dict[str, Any]:
        """Convert datetime fields to ISO strings."""
        if isinstance(data, dict):
            for key, value in data.items():
                if isinstance(value, datetime):
                    data[key] = value.isoformat()
                elif isinstance(value, dict):
                    data[key] = self._serialize_datetime_fields(value)
                elif isinstance(value, list):
                    data[key] = [self._serialize_datetime_fields(item) if isinstance(item, dict) else item for item in value]
        return data
    
    def _deserialize_datetime_fields(self, data: Dict[str, Any]) -> Dict[str, Any]:
        """Convert ISO strings back to datetime fields."""
        datetime_fields = ["created_at", "updated_at", "timestamp"]
        
        for field in datetime_fields:
            if field in data and isinstance(data[field], str):
                try:
                    data[field] = datetime.fromisoformat(data[field])
                except ValueError:
                    pass  # Keep as string if parsing fails
        
        # Handle nested datetime fields
        if "version_history" in data and isinstance(data["version_history"], list):
            for version in data["version_history"]:
                if isinstance(version, dict) and "timestamp" in version:
                    try:
                        version["timestamp"] = datetime.fromisoformat(version["timestamp"])
                    except ValueError:
                        pass
        
        return data
    
    def _deserialize_enum_fields(self, data: Dict[str, Any]) -> Dict[str, Any]:
        """Convert enum fields from strings."""
        if "category" in data and isinstance(data["category"], str):
            try:
                data["category"] = TemplateCategory(data["category"])
            except ValueError:
                data["category"] = TemplateCategory.CUSTOM
        
        if "status" in data and isinstance(data["status"], str):
            try:
                data["status"] = TemplateStatus(data["status"])
            except ValueError:
                data["status"] = TemplateStatus.DRAFT
        
        return data


class TemplateStorage:
    """Template storage management."""
    
    def __init__(self, storage_path: str = "/tmp/cloudprinter/templates"):
        """Initialize template storage.
        
        Args:
            storage_path: Base path for template storage
        """
        self.storage_path = Path(storage_path)
        self.templates_path = self.storage_path / "templates"
        self.backups_path = self.storage_path / "backups"
        self.cache_path = self.storage_path / "cache"
        self.exports_path = self.storage_path / "exports"
        
        # Create directory structure
        self._create_directory_structure()
        
        # Initialize index
        self.index_file = self.storage_path / "index.json"
        self.template_index = self._load_index()
    
    def _create_directory_structure(self):
        """Create storage directory structure."""
        directories = [
            self.templates_path / "network",
            self.templates_path / "system", 
            self.templates_path / "application",
            self.templates_path / "security",
            self.templates_path / "composite",
            self.templates_path / "custom",
            self.backups_path,
            self.cache_path,
            self.exports_path
        ]
        
        for directory in directories:
            directory.mkdir(parents=True, exist_ok=True)
    
    def _load_index(self) -> Dict[str, Dict[str, Any]]:
        """Load template index."""
        if self.index_file.exists():
            try:
                with open(self.index_file, 'r', encoding='utf-8') as f:
                    return json.load(f)
            except Exception as e:
                logger.error(f"Failed to load template index: {e}")
        
        return {}
    
    def _save_index(self):
        """Save template index."""
        try:
            with open(self.index_file, 'w', encoding='utf-8') as f:
                json.dump(self.template_index, f, indent=2, ensure_ascii=False)
        except Exception as e:
            logger.error(f"Failed to save template index: {e}")
    
    def _get_template_file_path(self, template: ConfigTemplate) -> Path:
        """Get file path for template."""
        category_dir = self.templates_path / template.category.value
        return category_dir / f"{template.template_id}.json"
    
    def save_template(self, template: ConfigTemplate) -> bool:
        """Save template to storage.
        
        Args:
            template: Template to save
            
        Returns:
            True if saved successfully
        """
        try:
            # Save template file
            template_file = self._get_template_file_path(template)
            
            # Create backup if template exists
            if TEMPLATE_CONFIG["backup_enabled"] and template_file.exists():
                self._create_backup(template.template_id)
            
            # Serialize template
            serializer = TemplateSerializer()
            template_data = serializer.export_template(
                template, format="json", 
                compress=TEMPLATE_CONFIG["compression"]
            )
            
            # Write to file
            with open(template_file, 'wb') as f:
                f.write(template_data)
            
            # Update index
            self.template_index[template.template_id] = {
                "name": template.name,
                "category": template.category.value,
                "version": template.version,
                "created_at": template.created_at.isoformat(),
                "updated_at": template.updated_at.isoformat(),
                "status": template.status.value,
                "file_path": str(template_file),
                "file_size": len(template_data),
                "usage_count": template.usage_count
            }
            
            self._save_index()
            
            logger.info(f"Template saved: {template.template_id}")
            return True
            
        except Exception as e:
            logger.error(f"Failed to save template {template.template_id}: {e}")
            return False
    
    def load_template(self, template_id: str) -> Optional[ConfigTemplate]:
        """Load template from storage.
        
        Args:
            template_id: Template identifier
            
        Returns:
            Loaded template or None
        """
        try:
            if template_id not in self.template_index:
                return None
            
            # Get file path from index
            file_path = Path(self.template_index[template_id]["file_path"])
            
            if not file_path.exists():
                logger.warning(f"Template file not found: {file_path}")
                return None
            
            # Read and deserialize
            with open(file_path, 'rb') as f:
                template_data = f.read()
            
            serializer = TemplateSerializer()
            template = serializer.import_template(
                template_data, format="json",
                compressed=TEMPLATE_CONFIG["compression"]
            )
            
            return template
            
        except Exception as e:
            logger.error(f"Failed to load template {template_id}: {e}")
            return None
    
    def delete_template(self, template_id: str) -> bool:
        """Delete template from storage.
        
        Args:
            template_id: Template identifier
            
        Returns:
            True if deleted successfully
        """
        try:
            if template_id not in self.template_index:
                return False
            
            # Create backup before deletion
            if TEMPLATE_CONFIG["backup_enabled"]:
                self._create_backup(template_id)
            
            # Get file path and delete
            file_path = Path(self.template_index[template_id]["file_path"])
            if file_path.exists():
                file_path.unlink()
            
            # Remove from index
            del self.template_index[template_id]
            self._save_index()
            
            logger.info(f"Template deleted: {template_id}")
            return True
            
        except Exception as e:
            logger.error(f"Failed to delete template {template_id}: {e}")
            return False
    
    def list_templates(self, category: Optional[TemplateCategory] = None,
                      status: Optional[TemplateStatus] = None,
                      tags: Optional[List[str]] = None) -> List[Dict[str, Any]]:
        """List templates with optional filters.
        
        Args:
            category: Filter by category
            status: Filter by status
            tags: Filter by tags
            
        Returns:
            List of template metadata
        """
        templates = []
        
        for template_id, metadata in self.template_index.items():
            # Apply filters
            if category and metadata["category"] != category.value:
                continue
            
            if status and metadata["status"] != status.value:
                continue
            
            # For tag filtering, we need to load the template
            if tags:
                template = self.load_template(template_id)
                if not template or not any(tag in template.tags for tag in tags):
                    continue
            
            templates.append({
                "template_id": template_id,
                **metadata
            })
        
        return templates
    
    def _create_backup(self, template_id: str):
        """Create backup of template."""
        try:
            if template_id not in self.template_index:
                return
            
            source_path = Path(self.template_index[template_id]["file_path"])
            if not source_path.exists():
                return
            
            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
            backup_name = f"{template_id}_{timestamp}.backup"
            backup_path = self.backups_path / backup_name
            
            shutil.copy2(source_path, backup_path)
            
        except Exception as e:
            logger.warning(f"Failed to create backup for {template_id}: {e}")


class TemplateManager:
    """Main template management system."""
    
    def __init__(self, storage_path: str = "/tmp/cloudprinter/templates"):
        """Initialize template manager.
        
        Args:
            storage_path: Storage path for templates
        """
        self.storage = TemplateStorage(storage_path)
        self.cache = TemplateCache(
            max_size=TEMPLATE_CONFIG["cache_size"],
            ttl=TEMPLATE_CONFIG["cache_ttl"]
        )
        self.validator = TemplateValidator()
        self.serializer = TemplateSerializer()
        self.executor = ThreadPoolExecutor(max_workers=4)
        
        logger.info("TemplateManager initialized")
    
    async def create_template(self, name: str, description: str = "",
                            category: TemplateCategory = TemplateCategory.CUSTOM,
                            config_schema: Optional[Dict[str, Any]] = None,
                            default_values: Optional[Dict[str, Any]] = None,
                            validation_rules: Optional[List[ValidationRule]] = None,
                            author: str = "system") -> str:
        """Create new configuration template.
        
        Args:
            name: Template name
            description: Template description
            category: Template category
            config_schema: Configuration schema
            default_values: Default configuration values
            validation_rules: Validation rules
            author: Template author
            
        Returns:
            Template ID
        """
        try:
            template_id = str(uuid.uuid4())
            
            template = ConfigTemplate(
                template_id=template_id,
                name=name,
                description=description,
                category=category,
                config_schema=config_schema or {},
                default_values=default_values or {},
                validation_rules=validation_rules or [],
                author=author,
                created_at=datetime.now(),
                updated_at=datetime.now(),
                status=TemplateStatus.DRAFT
            )
            
            # Validate template
            if TEMPLATE_CONFIG["auto_validation"]:
                validation_result = await self.validate_template(template)
                if not validation_result.is_valid:
                    raise ValueError(f"Template validation failed: {validation_result.errors}")
            
            # Save template
            success = await asyncio.get_event_loop().run_in_executor(
                self.executor, self.storage.save_template, template
            )
            
            if not success:
                raise Exception("Failed to save template")
            
            # Cache template
            self.cache.put(template)
            
            logger.info(f"Template created: {template_id}")
            return template_id
            
        except Exception as e:
            logger.error(f"Failed to create template: {e}")
            raise
    
    async def get_template(self, template_id: str) -> Optional[ConfigTemplate]:
        """Get template by ID.
        
        Args:
            template_id: Template identifier
            
        Returns:
            Template or None if not found
        """
        try:
            # Try cache first
            template = self.cache.get(template_id)
            if template:
                return template
            
            # Load from storage
            template = await asyncio.get_event_loop().run_in_executor(
                self.executor, self.storage.load_template, template_id
            )
            
            if template:
                # Cache template
                self.cache.put(template)
            
            return template
            
        except Exception as e:
            logger.error(f"Failed to get template {template_id}: {e}")
            return None
    
    async def update_template(self, template_id: str, 
                            updates: Dict[str, Any]) -> bool:
        """Update existing template.
        
        Args:
            template_id: Template identifier
            updates: Fields to update
            
        Returns:
            True if updated successfully
        """
        try:
            template = await self.get_template(template_id)
            if not template:
                return False
            
            if template.is_locked:
                raise ValueError("Template is locked for editing")
            
            # Create new version
            old_version = template.version
            version_parts = template.version.split('.')
            patch = int(version_parts[2]) + 1
            new_version = f"{version_parts[0]}.{version_parts[1]}.{patch}"
            
            # Record version history
            version_record = TemplateVersion(
                version=old_version,
                timestamp=template.updated_at,
                author=template.author,
                changes=[f"Updated to version {new_version}"],
                config_diff=updates
            )
            
            template.version_history.append(version_record)
            
            # Limit version history
            if len(template.version_history) > TEMPLATE_CONFIG["version_history_limit"]:
                template.version_history = template.version_history[-TEMPLATE_CONFIG["version_history_limit"]:]
            
            # Apply updates
            for field, value in updates.items():
                if hasattr(template, field):
                    setattr(template, field, value)
            
            template.version = new_version
            template.updated_at = datetime.now()
            
            # Validate updated template
            if TEMPLATE_CONFIG["auto_validation"]:
                validation_result = await self.validate_template(template)
                if not validation_result.is_valid:
                    raise ValueError(f"Template validation failed: {validation_result.errors}")
            
            # Save updated template
            success = await asyncio.get_event_loop().run_in_executor(
                self.executor, self.storage.save_template, template
            )
            
            if success:
                # Update cache
                self.cache.put(template)
                logger.info(f"Template updated: {template_id}")
            
            return success
            
        except Exception as e:
            logger.error(f"Failed to update template {template_id}: {e}")
            return False
    
    async def delete_template(self, template_id: str) -> bool:
        """Delete template.
        
        Args:
            template_id: Template identifier
            
        Returns:
            True if deleted successfully
        """
        try:
            success = await asyncio.get_event_loop().run_in_executor(
                self.executor, self.storage.delete_template, template_id
            )
            
            if success:
                # Remove from cache
                self.cache.invalidate(template_id)
                logger.info(f"Template deleted: {template_id}")
            
            return success
            
        except Exception as e:
            logger.error(f"Failed to delete template {template_id}: {e}")
            return False
    
    async def list_templates(self, category: Optional[TemplateCategory] = None,
                           status: Optional[TemplateStatus] = None,
                           tags: Optional[List[str]] = None) -> List[Dict[str, Any]]:
        """List templates with filters.
        
        Args:
            category: Filter by category
            status: Filter by status  
            tags: Filter by tags
            
        Returns:
            List of template metadata
        """
        try:
            templates = await asyncio.get_event_loop().run_in_executor(
                self.executor, self.storage.list_templates, category, status, tags
            )
            
            return templates
            
        except Exception as e:
            logger.error(f"Failed to list templates: {e}")
            return []
    
    async def validate_template(self, template: ConfigTemplate) -> ValidationResult:
        """Validate template configuration.
        
        Args:
            template: Template to validate
            
        Returns:
            Validation result
        """
        try:
            return await asyncio.get_event_loop().run_in_executor(
                self.executor, self.validator.validate_template, template
            )
        except Exception as e:
            logger.error(f"Template validation error: {e}")
            return ValidationResult(is_valid=False, errors=[str(e)])
    
    async def export_template(self, template_id: str, format: str = "json",
                            compress: bool = False) -> Optional[bytes]:
        """Export template to bytes.
        
        Args:
            template_id: Template identifier
            format: Export format
            compress: Whether to compress
            
        Returns:
            Exported template data or None
        """
        try:
            template = await self.get_template(template_id)
            if not template:
                return None
            
            return await asyncio.get_event_loop().run_in_executor(
                self.executor, self.serializer.export_template, 
                template, format, compress
            )
            
        except Exception as e:
            logger.error(f"Failed to export template {template_id}: {e}")
            return None
    
    async def import_template(self, data: bytes, format: str = "json",
                            compressed: bool = False, 
                            overwrite: bool = False) -> Optional[str]:
        """Import template from bytes.
        
        Args:
            data: Template data
            format: Data format
            compressed: Whether data is compressed
            overwrite: Whether to overwrite existing template
            
        Returns:
            Template ID or None if failed
        """
        try:
            template = await asyncio.get_event_loop().run_in_executor(
                self.executor, self.serializer.import_template,
                data, format, compressed
            )
            
            # Check if template already exists
            existing_template = await self.get_template(template.template_id)
            if existing_template and not overwrite:
                # Generate new ID
                template.template_id = str(uuid.uuid4())
                template.name += " (Imported)"
            
            # Validate imported template
            validation_result = await self.validate_template(template)
            if not validation_result.is_valid:
                logger.warning(f"Imported template has validation issues: {validation_result.errors}")
                # Continue with import but mark as draft
                template.status = TemplateStatus.DRAFT
            
            # Save template
            success = await asyncio.get_event_loop().run_in_executor(
                self.executor, self.storage.save_template, template
            )
            
            if success:
                self.cache.put(template)
                logger.info(f"Template imported: {template.template_id}")
                return template.template_id
            
            return None
            
        except Exception as e:
            logger.error(f"Failed to import template: {e}")
            return None
    
    async def get_statistics(self) -> Dict[str, Any]:
        """Get template manager statistics.
        
        Returns:
            Statistics dictionary
        """
        try:
            templates = await self.list_templates()
            
            stats = {
                "template_manager": {
                    "total_templates": len(templates),
                    "cache_size": len(self.cache.cache),
                    "cache_hit_ratio": 0.0,  # Would need to track this
                    "templates_by_category": {},
                    "templates_by_status": {},
                    "most_used_templates": []
                }
            }
            
            # Count by category and status
            for template_meta in templates:
                category = template_meta["category"]
                status = template_meta["status"]
                
                stats["template_manager"]["templates_by_category"][category] = \
                    stats["template_manager"]["templates_by_category"].get(category, 0) + 1
                
                stats["template_manager"]["templates_by_status"][status] = \
                    stats["template_manager"]["templates_by_status"].get(status, 0) + 1
            
            # Get most used templates
            templates_with_usage = [(t["template_id"], t["name"], t["usage_count"]) 
                                   for t in templates if "usage_count" in t]
            templates_with_usage.sort(key=lambda x: x[2], reverse=True)
            stats["template_manager"]["most_used_templates"] = templates_with_usage[:5]
            
            return stats
            
        except Exception as e:
            logger.error(f"Failed to get statistics: {e}")
            return {"template_manager": {"error": str(e)}}
    
    async def cleanup(self):
        """Clean up template manager resources."""
        self.cache.clear()
        self.executor.shutdown(wait=True)
        logger.info("TemplateManager cleanup completed")


# Example usage and testing
if __name__ == "__main__":
    import asyncio
    import tempfile
    
    async def test_template_manager():
        """Test template manager functionality."""
        with tempfile.TemporaryDirectory() as temp_dir:
            # Create template manager
            template_manager = TemplateManager(temp_dir)
            
            try:
                # Test template creation
                print("=== Testing Template Creation ===")
                
                config_schema = {
                    "wifi": {
                        "type": "object",
                        "properties": {
                            "ssid": {"type": "string"},
                            "password": {"type": "string"},
                            "security": {"type": "string"}
                        }
                    }
                }
                
                default_values = {
                    "wifi": {
                        "security": "WPA2",
                        "hidden": False
                    }
                }
                
                validation_rules = [
                    ValidationRule(
                        field_path="wifi.ssid",
                        rule_type="length",
                        rule_params={"min": 1, "max": 32},
                        error_message="SSID must be 1-32 characters"
                    )
                ]
                
                template_id = await template_manager.create_template(
                    name="WiFi Configuration Template",
                    description="Template for WiFi network configuration",
                    category=TemplateCategory.NETWORK,
                    config_schema=config_schema,
                    default_values=default_values,
                    validation_rules=validation_rules
                )
                
                print(f"Created template: {template_id}")
                
                # Test template retrieval
                print("\n=== Testing Template Retrieval ===")
                template = await template_manager.get_template(template_id)
                if template:
                    print(f"Retrieved template: {template.name}")
                    print(f"Category: {template.category.value}")
                    print(f"Version: {template.version}")
                
                # Test template update
                print("\n=== Testing Template Update ===")
                updates = {
                    "description": "Updated WiFi configuration template",
                    "tags": ["wifi", "network", "configuration"]
                }
                
                success = await template_manager.update_template(template_id, updates)
                print(f"Template update: {'success' if success else 'failed'}")
                
                # Test template listing
                print("\n=== Testing Template Listing ===")
                templates = await template_manager.list_templates(
                    category=TemplateCategory.NETWORK
                )
                print(f"Found {len(templates)} network templates")
                
                # Test template validation
                print("\n=== Testing Template Validation ===")
                template = await template_manager.get_template(template_id)
                if template:
                    validation_result = await template_manager.validate_template(template)
                    print(f"Template valid: {validation_result.is_valid}")
                    if validation_result.errors:
                        print(f"Errors: {validation_result.errors}")
                    if validation_result.warnings:
                        print(f"Warnings: {validation_result.warnings}")
                
                # Test template export/import
                print("\n=== Testing Template Export/Import ===")
                exported_data = await template_manager.export_template(
                    template_id, format="json", compress=True
                )
                
                if exported_data:
                    print(f"Exported template size: {len(exported_data)} bytes")
                    
                    # Import as new template
                    imported_id = await template_manager.import_template(
                        exported_data, format="json", compressed=True
                    )
                    
                    if imported_id:
                        print(f"Imported template: {imported_id}")
                
                # Test statistics
                print("\n=== Testing Statistics ===")
                stats = await template_manager.get_statistics()
                print(f"Statistics: {json.dumps(stats, indent=2)}")
                
            finally:
                await template_manager.cleanup()
    
    # Run test
    asyncio.run(test_template_manager())